package com.zhucx.thread;

import com.ruoyi.common.utils.StringUtils;
import com.zhucx.ZhuCrawler;
import com.zhucx.exception.XxlCrawlerException;
import com.zhucx.model.PageRequest;
import com.zhucx.parser.strategy.NonPageParser;
import com.zhucx.utils.JsoupUtil;
import com.zhucx.utils.UrlUtil;
import org.apache.commons.lang3.ObjectUtils;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;

import java.io.IOException;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.TimeUnit;

/**
 * crawler thread
 *
 */
public class CrawlerThread implements Runnable {
    @Autowired
    JsoupUtil jsoupUtil;
    private static Logger logger = LoggerFactory.getLogger(CrawlerThread.class);

    private ZhuCrawler crawler;
    private boolean running;
    private boolean toStop;
    public CrawlerThread(ZhuCrawler crawler) {
        this.crawler = crawler;
        this.running = true;
        this.toStop = false;
    }
    public void toStop() {
        this.toStop = true;
    }
    public boolean isRunning() {
        return running;
    }

    @Override
    public void run() {

        while (!toStop) {
            try {
                // ------- url ----------
                running = false;
                //TODO 终止
                crawler.tryFinish();
                String link = crawler.getRunData().getUrl();
                running = true;
                logger.info(">>>>>>>>>>> zhuzi crawler, process link : {}", link);
                if (!UrlUtil.isUrl(link)) {
                    continue;
                }

                // failover
                for (int i = 0; i < (1 + crawler.getRunConf().getFailRetryCount()); i++) {

                    boolean ret = false;
                    try {
                        // make request
                        PageRequest pageRequest = makePageRequest(link);

                        // pre parse
                        crawler.getRunConf().getPageParser().preParse(pageRequest);

                        // parse
                        if (crawler.getRunConf().getPageParser() instanceof NonPageParser) {
                            ret = processNonPage(pageRequest);
                        } else {
                            ret = processPage(pageRequest);
                        }
                    } catch (Throwable e) {
                        logger.info(">>>>>>>>>>> zhuzi crawler proocess error.", e);
                    }

                    if (crawler.getRunConf().getPauseMillis() > 0) {
                        try {
                            TimeUnit.MILLISECONDS.sleep(crawler.getRunConf().getPauseMillis());
                        } catch (InterruptedException e) {
                            logger.info(">>>>>>>>>>> zhuzi crawler thread is interrupted. 2{}", e.getMessage());
                        }
                    }
                    if (ret) {
                        break;
                    }
                }

            } catch (Throwable e) {
                if (e instanceof InterruptedException) {
                    logger.info(">>>>>>>>>>> zhuzi crawler thread is interrupted. {}", e.getMessage());
                } else if (e instanceof XxlCrawlerException) {
                    logger.info(">>>>>>>>>>> zhuzi crawler thread {}", e.getMessage());
                } else {
                    logger.error(e.getMessage(), e);
                }
            }

        }
    }

    /**
     * 采集 请求参数
     *
     * @param link
     * @return PageRequest
     */
    private PageRequest makePageRequest(String link){
        String userAgent = crawler.getRunConf().getUserAgentList().size()>1
                ?crawler.getRunConf().getUserAgentList().get(new Random().nextInt(crawler.getRunConf().getUserAgentList().size()))
                :crawler.getRunConf().getUserAgentList().size()==1?crawler.getRunConf().getUserAgentList().get(0):null;
//        Proxy proxy = null;
//        if (crawler.getRunConf().getProxyMaker() != null) {
//            proxy = crawler.getRunConf().getProxyMaker().make();
//        }

        PageRequest pageRequest = new PageRequest();
        pageRequest.setUrl(link);
        pageRequest.setParamMap(crawler.getRunConf().getParamMap());
        pageRequest.setCookieMap(crawler.getRunConf().getCookieMap());
        pageRequest.setHeaderMap(crawler.getRunConf().getHeaderMap());
        pageRequest.setUserAgent(userAgent);
        //hzb++
        pageRequest.setSelectElement(crawler.getRunConf().getSelectElement());
        pageRequest.setReferrer(crawler.getRunConf().getReferrer());
        pageRequest.setIfPost(crawler.getRunConf().isIfPost());
        pageRequest.setTimeoutMillis(crawler.getRunConf().getTimeoutMillis());
//        pageRequest.setProxy(proxy);
        pageRequest.setValidateTLSCertificates(crawler.getRunConf().isValidateTLSCertificates());

        return pageRequest;
    }

    /**
     * process non page
     * @param pageRequest
     * @return boolean
     */
    private boolean processNonPage(PageRequest pageRequest){
        NonPageParser nonPageParser = (NonPageParser) crawler.getRunConf().getPageParser();

        String pagesource = jsoupUtil.loadPageSource(pageRequest);
        if (pagesource == null) {
            return false;
        }
        nonPageParser.parse(pageRequest.getUrl(), pagesource);
        return true;
    }

    /**
     * process page
     * @param pageRequest
     * @return boolean
     */
    private boolean processPage(PageRequest pageRequest) throws IllegalAccessException, InstantiationException, IOException, InterruptedException {
       //加载界面解析器
        Document html = crawler.getRunConf().getPageLoader().load(pageRequest);

        if (html == null) {
            return false;
        }

        // ------- child link list (FIFO队列,广度优先) ----------
        if (crawler.getRunConf().isAllowSpread()) {     // limit child spread
            Set<String> links = jsoupUtil.findLinks(html);
            if (links != null && links.size() > 0) {
                for (String item : links) {
                    if (crawler.getRunConf().validWhiteUrl(item)) {      // limit unvalid-child spread
                        crawler.getRunData().addUrl(item);
                    }
                }
            }
        }

        // ------- pagevo ----------
        if (!crawler.getRunConf().validWhiteUrl(pageRequest.getUrl())) {     // limit unvalid-page parse, only allow spread child, finish here
            return true;
        }

        // pagevo class-field info
//        Class pageVoClassType = Object.class;

//        Type pageVoParserClass = crawler.getRunConf().getPageParser().getClass().getGenericSuperclass();
//        if (pageVoParserClass instanceof ParameterizedType) {
//            Type[] pageVoClassTypes = ((ParameterizedType)pageVoParserClass).getActualTypeArguments();
//            pageVoClassType = (Class) pageVoClassTypes[0];
//        }
//
//        PageSelect pageVoSelect = (PageSelect) pageVoClassType.getAnnotation(PageSelect.class);
//        String pageVoCssQuery = (pageVoSelect!=null && pageVoSelect.cssQuery()!=null && pageVoSelect.cssQuery().trim().length()>0)?pageVoSelect.cssQuery():"html";

        // pagevo document 2 object

        String selectEl = pageRequest.getSelectElement()==null?"html":pageRequest.getSelectElement();
        String deleteEl = pageRequest.getDeleteElements();
        html = deleteDiv(html,deleteEl);
        if(ObjectUtils.isEmpty(selectEl)){
            return false;
        }
        Elements pageVoElements = new Elements();
        String divlsit[] = selectEl.split(";");
        Elements els = new Elements();
        for(String div:divlsit){
            els = html.select(div);
            for(Element el:els){
                pageVoElements.add(el);
            }
        }
        if (pageVoElements != null && pageVoElements.hasText()) {
            for (Element pageVoElement : pageVoElements) {
                // pagevo output

                crawler.getRunConf().getPageParser().parse(html, pageVoElement);
            }
        }

        return true;
    }

    /**
     * 删除标签
     * @param doc
     * @param deleteDiv
     * @return
     */
    public static Document deleteDiv(Document doc,String deleteDiv){
        if(StringUtils.isEmpty(deleteDiv)){
            return doc;
        }
        String divlsit[] = deleteDiv.split(";");
        for(String div:divlsit){
            try {
                doc.select(div).remove();
            }catch (Exception e){
                logger.error("删除div失败："+e.getMessage());
            }
        }
        return doc;
    }

}